'''
Created on Sep 16, 2012

@author: kingsfield
'''
from collections import defaultdict
from util import Util
from datetime import datetime

trainfile = '/home/kingsfield/Desktop/BestBuyPc/old_train.csv'
testfile = '/home/kingsfield/Desktop/BestBuyPc/old_test.csv'
outfile = '/home/kingsfield/Desktop/BestBuyPc/words_spell_test.csv'

def get_date(click_time):
    try:
        t = datetime.strptime(click_time, '"%Y-%m-%d %H:%M:%S.%f"')
    except:
        t = datetime.strptime(click_time, '"%Y-%m-%d %H:%M:%S"')
    return t

if __name__ == '__main__':
    with open(trainfile) as fr:
        data = fr.readlines()
    fw = open(outfile, 'w')
    uids = [line.split(',')[0] for line in data]
    raw_query = [line.split(',')[3] for line in data]
    skus = [line.split(',')[1] for line in data]
    word_train = set()
    for idx in xrange(len(data)):
        q = raw_query[idx]
        words = Util.wordutil.get_correct_words(q)
        for w in words:
            word_train.add(w)
        if len(words) == 0:
            print '[train]:uid=%s,sku=%s,query=%s' % (uids[idx], skus[idx], q)
#    
#    with open(trainfile) as fr:
#        data = fr.readlines()
#    data = data[1:]
#    query_t = [get_date(line.split(',')[4]) for line in data]
#    max_t = max(query_t)
#    min_t = min(query_t)
#    print min_t , max_t
    
    train_uid = set(uids)
    uid_count = defaultdict(int)
    for uid in uids:
        uid_count[uid] += 1
    rank = sorted(uid_count.items(), key=lambda x:x[1], reverse=True)
    hot_uid = [i for i in rank if i[1] >= 5]
    print len(hot_uid)
    #print hot_uid

    with open(testfile) as fr:
        data = fr.readlines()
    uids = [line.split(',')[0] for line in data]
    test_uid = set(uids)
    
    raw_query = [line.split(',')[2] for line in data]
    no_exist_count = 0
    for idx in xrange(len(data)):
        q = raw_query[idx]
        words = Util.wordutil.get_correct_words(q)
        noexistFlag = False
        for w in words:
            if w not in word_train:
                #print 'no exist word: %s' % w
                noexistFlag = True
        if noexistFlag:
            no_exist_count += 1
            print 'idx=%s,query=%s, words=%s' % (idx, q, words)
            fw.write(' '.join(words) + ',\n')
#        if len(words) == 0:
#            print '[test]:uid=%s,sku=%s,query=%s' % (uids[idx], skus[idx], q)
    print 'no_exist_count=%s' % no_exist_count
    #print len(test_uid.intersection(train_uid))
    print len(set([i[0] for i in hot_uid]).intersection(test_uid))
    #print set([i[0] for i in hot_uid]).intersection(test_uid)
